#!/bin/bash
###
# auther:hongxing.fan
# description:put log file on hadoop
# date:2015-03-17 Tuesday
# file:Put_reg.sh

operate_date=`date  -d "-1 day" +%Y%m%d`
if [[ $# -eq 1 ]]
then
	operate_date=$1
fi
local_path="/home/work/workToHadoop/reg_log"
hdfs_path="/user/hive/warehouse/pass_data.db/reg_log"
# $1=$? $2=operate $3=source
stat(){
	end_time=`date '+%Y-%m-%d %H:%M:%S'`
	if [[ $1 -eq 0 ]];then
		echo "$2	$end_time	$3	$4	$5	$6	$7	$8	$9	success" >> $log
	else
		echo "$2	$end_time	$3	$4	$5	$6	$7	$8	$9	error" >> $log
		return 1
	fi
	return 0
}
source ~/.bashrc

cd $local_path
log="/home/work/workToHadoop/log/${operate_date}.log"
put_reg="/home/work/workToHadoop/log/put_reg_${operate_date}.txt"
get_reg="/home/work/workToHadoop/log/get_reg_${operate_date}.txt"
rm $put_reg
touch $put_reg
host=$(hostname)
will_put=$(grep -vxFf $put_reg $get_reg)
len=${#will_put[0]}
pids=$(ps axu | grep -E "Get_reg.sh" | wc -l | awk '{print $1}')
while [[ $pids -gt 1 || $len -ne 0 ]]
do
	for f in ${will_put[0]}
	do
		dir=$(echo $f | awk -F"att" '{print $1}')
		file=$(echo $f | awk -F"dt=$operate_date/" '{print $2}')
		`hdfs dfs -mkdir -p $hdfs_path/dt=$operate_date`
		begin_time=`date '+%Y-%m-%d %H:%M:%S'`
		`hadoop dfs -put $f $hdfs_path/dt=$operate_date`
		stat $? "$begin_time" "reg_log" "$operate_date" "$host" "put" "" "$hdfs_path/dt=$operate_date/$file" "$f"
		echo "$f" >> $put_reg
		$(hive -e "use pass_data;alter table reg_log add partition(dt=$operate_date) location 'hdfs:///user/hive/warehouse/pass_data.db/reg_log/dt=$operate_date';")
	done
	sleep 1
	pids=$(ps axu | grep -E "Get_reg.sh" | wc -l | awk '{print $1}')
	will_put=$(grep -vxFf $put_reg $get_reg)
	len=${#will_put[0]}
done
